import requests
from lxml import etree



class Biquge:

    def __init__(self):
        # 制造请求头，将爬虫伪装成浏览器
        self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36'}

    def c_downloader(self,url,title):
        '''
        章节下载器
        url 章节地址
        title 小说名
        '''

        response = requests.get(url,headers=self.headers)
        response.encoding = 'utf-8'
        # 转换成etree格式
        selector = etree.HTML(response.text)
        # 获取章节的标题
        ctitle = selector.xpath('//*[@class="content"]/h1/text()')[0]
        print('正在下载章节——%s...'%ctitle)

        # 获取章节的内容
        ccontent = selector.xpath('//*[@id="content"]/text()')
        # s = ''
        # for line in ccontent:
        #     s += line
        ccontent = ''.join(ccontent)

        # 保存
        with open(r'python\day08\%s.txt'%title,'a',encoding='utf-8') as file:
            file.write(ctitle+'\n'+ccontent+'\n')

    def n_downloader(self,url):
        '''
        获取所有的章节地址
        url 小说的地址
        '''

        response = requests.get(url,headers=self.headers)
        # 指定编码集
        response.encoding = 'utf-8'
        # 转换为etree
        selector = etree.HTML(response.text)

        # 获取小说名
        title = selector.xpath('//h2/text()')[0]
        print('开始下载小说《%s》...'%title)

        # 获取所有的章节地址
        urls = selector.xpath('//*[@class="listmain"]/dl/dd/a/@href')
        # 补全地址
        urls = ['https://www.biqumo.com'+url for url in urls]
        # 遍历章节列表，下载所有内容
        for url in urls[0:5]:
            self.c_downloader(url,title)

    def downloader(self):
        response = requests.get('https://www.biqumo.com/wanben/',headers=self.headers)
        response.encoding = 'utf-8'
        selector = etree.HTML(response.text)

        # 获取所有的全本小说的列表
        urls = selector.xpath('//*[@id="main"]/div[1]/ul/li/span[2]/a/@href')
        # 补全地址
        urls = ['https://www.biqumo.com'+url for url in urls]
        for url in urls:
            self.n_downloader(url)
            
if __name__ == "__main__":
    Biquge().downloader()
